<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Pipeline Aggregations | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="search-aggregations.html" title="Aggregations">
<link rel="prev" href="search-aggregations-bucket-range-field-note.html" title="Subtleties of bucketing range fields">
<link rel="next" href="search-aggregations-pipeline-bucket-script-aggregation.html" title="Bucket Script Aggregation">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="search-aggregations.html">Aggregations</a></span>
»
<span class="breadcrumb-node">Pipeline Aggregations</span>
</div>
<div class="navheader">
<span class="prev">
<a href="search-aggregations-bucket-range-field-note.html">« Subtleties of bucketing range fields</a>
</span>
<span class="next">
<a href="search-aggregations-pipeline-bucket-script-aggregation.html">Bucket Script Aggregation »</a>
</span>
</div>
<div class="chapter">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="search-aggregations-pipeline"></a>Pipeline Aggregations<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/aggregations/pipeline.asciidoc">edit</a>
</h2>
</div></div></div>
<p>Pipeline aggregations work on the outputs produced from other aggregations rather than from document sets, adding
information to the output tree. There are many different types of pipeline aggregation, each computing different information from
other aggregations, but these types can be broken down into two families:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<em>Parent</em>
</span>
</dt>
<dd>
A family of pipeline aggregations that is provided with the output of its parent aggregation and is able
to compute new buckets or new aggregations to add to existing buckets.
</dd>
<dt>
<span class="term">
<em>Sibling</em>
</span>
</dt>
<dd>
Pipeline aggregations that are provided with the output of a sibling aggregation and are able to compute a
new aggregation which will be at the same level as the sibling aggregation.
</dd>
</dl>
</div>
<p>Pipeline aggregations can reference the aggregations they need to perform their computation by using the <code class="literal">buckets_path</code>
parameter to indicate the paths to the required metrics. The syntax for defining these paths can be found in the
<a class="xref" href="search-aggregations-pipeline.html#buckets-path-syntax" title="buckets_path Syntax"><code class="literal">buckets_path</code> Syntax</a> section below.</p>
<p>Pipeline aggregations cannot have sub-aggregations but depending on the type it can reference another pipeline in the <code class="literal">buckets_path</code>
allowing pipeline aggregations to be chained.  For example, you can chain together two derivatives to calculate the second derivative
(i.e. a derivative of a derivative).</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Because pipeline aggregations only add to the output, when chaining pipeline aggregations the output of each pipeline aggregation
will be included in the final output.</p>
</div>
</div>
<h3>
<a id="buckets-path-syntax"></a><code class="literal">buckets_path</code> Syntax<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/aggregations/pipeline.asciidoc">edit</a>
</h3>
<p>Most pipeline aggregations require another aggregation as their input.  The input aggregation is defined via the <code class="literal">buckets_path</code>
parameter, which follows a specific format:</p>
<div class="pre_wrapper lang-ebnf">
<pre class="programlisting prettyprint lang-ebnf">AGG_SEPARATOR       =  `&gt;` ;
METRIC_SEPARATOR    =  `.` ;
AGG_NAME            =  &lt;the name of the aggregation&gt; ;
METRIC              =  &lt;the name of the metric (in case of multi-value metrics aggregation)&gt; ;
MULTIBUCKET_KEY     =  `[&lt;KEY_NAME&gt;]`
PATH                =  &lt;AGG_NAME&gt;&lt;MULTIBUCKET_KEY&gt;? (&lt;AGG_SEPARATOR&gt;, &lt;AGG_NAME&gt; )* ( &lt;METRIC_SEPARATOR&gt;, &lt;METRIC&gt; ) ;</pre>
</div>
<p>For example, the path <code class="literal">"my_bucket&gt;my_stats.avg"</code> will path to the <code class="literal">avg</code> value in the <code class="literal">"my_stats"</code> metric, which is
contained in the <code class="literal">"my_bucket"</code> bucket aggregation.</p>
<p>Paths are relative from the position of the pipeline aggregation; they are not absolute paths, and the path cannot go back "up" the
aggregation tree. For example, this moving average is embedded inside a date_histogram and refers to a "sibling"
metric <code class="literal">"the_sum"</code>:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /_search
{
    "aggs": {
        "my_date_histo":{
            "date_histogram":{
                "field":"timestamp",
                "calendar_interval":"day"
            },
            "aggs":{
                "the_sum":{
                    "sum":{ "field": "lemmings" } <a id="CO247-1"></a><i class="conum" data-value="1"></i>
                },
                "the_movavg":{
                    "moving_avg":{ "buckets_path": "the_sum" } <a id="CO247-2"></a><i class="conum" data-value="2"></i>
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/544.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO247-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The metric is called <code class="literal">"the_sum"</code></p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO247-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">buckets_path</code> refers to the metric via a relative path <code class="literal">"the_sum"</code></p>
</td>
</tr>
</table>
</div>
<p><code class="literal">buckets_path</code> is also used for Sibling pipeline aggregations, where the aggregation is "next" to a series of buckets
instead of embedded "inside" them.  For example, the <code class="literal">max_bucket</code> aggregation uses the <code class="literal">buckets_path</code> to specify
a metric embedded inside a sibling aggregation:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /_search
{
    "aggs" : {
        "sales_per_month" : {
            "date_histogram" : {
                "field" : "date",
                "calendar_interval" : "month"
            },
            "aggs": {
                "sales": {
                    "sum": {
                        "field": "price"
                    }
                }
            }
        },
        "max_monthly_sales": {
            "max_bucket": {
                "buckets_path": "sales_per_month&gt;sales" <a id="CO248-1"></a><i class="conum" data-value="1"></i>
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/545.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO248-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p><code class="literal">buckets_path</code> instructs this max_bucket aggregation that we want the maximum value of the <code class="literal">sales</code> aggregation in the
<code class="literal">sales_per_month</code> date histogram.</p>
</td>
</tr>
</table>
</div>
<p>If a Sibling pipeline agg references a multi-bucket aggregation, such as a <code class="literal">terms</code> agg, it also has the option to
select specific keys from the multi-bucket.  For example, a <code class="literal">bucket_script</code> could select two specific buckets (via
their bucket keys) to perform the calculation:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /_search
{
    "aggs" : {
        "sales_per_month" : {
            "date_histogram" : {
                "field" : "date",
                "calendar_interval" : "month"
            },
            "aggs": {
                "sale_type": {
                    "terms": {
                        "field": "type"
                    },
                    "aggs": {
                        "sales": {
                            "sum": {
                                "field": "price"
                            }
                        }
                    }
                },
                "hat_vs_bag_ratio": {
                    "bucket_script": {
                        "buckets_path": {
                            "hats": "sale_type['hat']&gt;sales", <a id="CO249-1"></a><i class="conum" data-value="1"></i>
                            "bags": "sale_type['bag']&gt;sales"  <a id="CO249-2"></a><i class="conum" data-value="1"></i>
                        },
                        "script": "params.hats / params.bags"
                    }
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/546.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO249-1"><i class="conum" data-value="1"></i></a><a href="#CO249-2"></a></p>
</td>
<td align="left" valign="top">
<p><code class="literal">buckets_path</code> selects the hats and bags buckets (via <code class="literal">['hat']</code>/<code class="literal">['bag']`</code>) to use in the script specifically,
instead of fetching all the buckets from <code class="literal">sale_type</code> aggregation</p>
</td>
</tr>
</table>
</div>
<h3>
<a id="_special_paths"></a>Special Paths<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/aggregations/pipeline.asciidoc">edit</a>
</h3>
<p>Instead of pathing to a metric, <code class="literal">buckets_path</code> can use a special <code class="literal">"_count"</code> path.  This instructs
the pipeline aggregation to use the document count as its input.  For example, a moving average can be calculated on the document count of each bucket, instead of a specific metric:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /_search
{
    "aggs": {
        "my_date_histo": {
            "date_histogram": {
                "field":"timestamp",
                "calendar_interval":"day"
            },
            "aggs": {
                "the_movavg": {
                    "moving_avg": { "buckets_path": "_count" } <a id="CO250-1"></a><i class="conum" data-value="1"></i>
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/547.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO250-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>By using <code class="literal">_count</code> instead of a metric name, we can calculate the moving average of document counts in the histogram</p>
</td>
</tr>
</table>
</div>
<p>The <code class="literal">buckets_path</code> can also use <code class="literal">"_bucket_count"</code> and path to a multi-bucket aggregation to use the number of buckets
returned by that aggregation in the pipeline aggregation instead of a metric. for example a <code class="literal">bucket_selector</code> can be
used here to filter out buckets which contain no buckets for an inner terms aggregation:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST /sales/_search
{
  "size": 0,
  "aggs": {
    "histo": {
      "date_histogram": {
        "field": "date",
        "calendar_interval": "day"
      },
      "aggs": {
        "categories": {
          "terms": {
            "field": "category"
          }
        },
        "min_bucket_selector": {
          "bucket_selector": {
            "buckets_path": {
              "count": "categories._bucket_count" <a id="CO251-1"></a><i class="conum" data-value="1"></i>
            },
            "script": {
              "source": "params.count != 0"
            }
          }
        }
      }
    }
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/548.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO251-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>By using <code class="literal">_bucket_count</code> instead of a metric name, we can filter out <code class="literal">histo</code> buckets where they contain no buckets
for the <code class="literal">categories</code> aggregation</p>
</td>
</tr>
</table>
</div>
<h3>
<a id="dots-in-agg-names"></a>Dealing with dots in agg names<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/aggregations/pipeline.asciidoc">edit</a>
</h3>
<p>An alternate syntax is supported to cope with aggregations or metrics which
have dots in the name, such as the <code class="literal">99.9</code>th
<a class="xref" href="search-aggregations-metrics-percentile-aggregation.html" title="Percentiles Aggregation">percentile</a>. This metric
may be referred to as:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"buckets_path": "my_percentile[99.9]"</pre>
</div>
<h3>
<a id="gap-policy"></a>Dealing with gaps in the data<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/aggregations/pipeline.asciidoc">edit</a>
</h3>
<p>Data in the real world is often noisy and sometimes contains <span class="strong strong"><strong>gaps</strong></span> — places where data simply doesn’t exist.  This can
occur for a variety of reasons, the most common being:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Documents falling into a bucket do not contain a required field
</li>
<li class="listitem">
There are no documents matching the query for one or more buckets
</li>
<li class="listitem">
The metric being calculated is unable to generate a value, likely because another dependent bucket is missing a value.
Some pipeline aggregations have specific requirements that must be met (e.g. a derivative cannot calculate a metric for the
first value because there is no previous value, HoltWinters moving average need "warmup" data to begin calculating, etc)
</li>
</ul>
</div>
<p>Gap policies are a mechanism to inform the pipeline aggregation about the desired behavior when "gappy" or missing
data is encountered.  All pipeline aggregations accept the <code class="literal">gap_policy</code> parameter.  There are currently two gap policies
to choose from:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<em>skip</em>
</span>
</dt>
<dd>
This option treats missing data as if the bucket does not exist.  It will skip the bucket and continue
calculating using the next available value.
</dd>
<dt>
<span class="term">
<em>insert_zeros</em>
</span>
</dt>
<dd>
This option will replace missing values with a zero (<code class="literal">0</code>) and pipeline aggregation computation will
proceed as normal.
</dd>
</dl>
</div>
















</div>
<div class="navfooter">
<span class="prev">
<a href="search-aggregations-bucket-range-field-note.html">« Subtleties of bucketing range fields</a>
</span>
<span class="next">
<a href="search-aggregations-pipeline-bucket-script-aggregation.html">Bucket Script Aggregation »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
